package com.chak.voyage.crawler;

import com.chak.voyage.crawler.pipeline.Article2MySQLPipeline;
import com.chak.voyage.crawler.processor.lvmama.LvmamaProcessor;
import com.chak.voyage.crawler.processor.lvmama.schedual.LvmamaScheduler;
import com.chak.voyage.service.DataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Created by delzh on 2016/9/4.
 */
@Component
public class Crawler {

    @Autowired
    private Article2MySQLPipeline article2MySQLPipeline;

    @Autowired
    private LvmamaScheduler lvmamaScheduler;

    public void crawl() {
        LvmamaProcessor lvmamaProcessor = new LvmamaProcessor();
        Spider articleSpider = Spider.create(lvmamaProcessor)
                .scheduler(lvmamaScheduler)
                .addRequest(new Request("http://www.lvmama.com/lvyou/").putExtra("type",0))
                .addRequest(new Request("http://www.lvmama.com/public/site_map").putExtra("type",0))
                .addPipeline(article2MySQLPipeline)
//                .addPipeline(new ConsolePipeline())
                .thread(15);

        ExecutorService executor = Executors.newFixedThreadPool(10);
        executor.execute(articleSpider);
    }
}
